{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://rate.tmall.com/list_detail_rate.htm?itemId=558760911386&spuId=877095771&sellerId=2616970884&order=3&currentPage=100\n"
     ]
    },
    {
     "ename": "JSONDecodeError",
     "evalue": "Expecting ',' delimiter: line 1 column 1234 (char 1233)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-e889c913a280>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     32\u001b[0m     \u001b[0mstartLoc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjsonData\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'{'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m     \u001b[0mjsonData\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjsonData\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstartLoc\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m     \u001b[0mjsonData\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjsonData\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     35\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[0muserId\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjsonData\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'rateDetail'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'rateList'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;31m#用户ID\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\__init__.py\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    317\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    318\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 319\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    320\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    321\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    337\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m         \"\"\"\n\u001b[1;32m--> 339\u001b[1;33m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    340\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    341\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\json\\decoder.py\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m    353\u001b[0m         \"\"\"\n\u001b[0;32m    354\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 355\u001b[1;33m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    356\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    357\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mJSONDecodeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Expecting value\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mJSONDecodeError\u001b[0m: Expecting ',' delimiter: line 1 column 1234 (char 1233)"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "import time\n",
    "import numpy as np\n",
    "from fake_useragent import UserAgent\n",
    "ua = UserAgent()\n",
    "import json\n",
    "url='https://rate.tmall.com/list_detail_rate.htm?itemId=558760911386&spuId=877095771&sellerId=2616970884&order=3&currentPage=1&append=0&content=1&tagId=&posi=&picture=0&groupId=&ua=098%23E1hvh9vnvpQvUvCkvvvvvjiPRLzWsjtbn2FpAjivPmPpzjimn2FvgjrUPFqOtjn8RphvCvvvphmCvpvWzPsocT5Nznswcoa43QhvCvmvphvPvpvhvv2MMQhCvvOvChCvvvmtvpvIvvCvpvvvvvvvvhXVvvmvAQvvByOvvUhQvvCVB9vv9BQvvhXVvvmCj8yCvv9vvhhIA2%2BtZgyCvvOCvhE2zW9EvpCW9jCxxB0UHdpLh09CdPpBR3VZJQ0HABvz5FUSLn7l%2BFwTdoD6q8ZsH2JZRl4gQfutnCB1pGsWVCuZHk62Qa7tnCpXwZ5w1E94eErABsyCvvpvvhCviQhvCvvvpZptvpvhvvCvpUhCvms9MniN6aMwznAakHIu6kOU4Ik1Qz%2BfivDi4dMWgFrivI8mkgl39o8mp98PFp%3D%3D&itemPropertyId=&itemPropertyIndex=&userPropertyId=&userPropertyIndex=&rateQuery=&location=&needFold=0&_ksTS=1551335588108_2327&callback=jsonp2328'\n",
    " \n",
    "headers={\n",
    "    'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',\n",
    "    'cookie':\"thw=cn; cna=ETbtE4ifXHQCAd9oA5dGIN3z; t=c59fb4290eb211b16fd60e1863380500; tg=0; enc=oOYGupIdaeGQt61OoQ%2BLY7oOgmozk7L8adzkcLaTZti2g3RBXH1OrtqGWKVeSLZJn2ZP37b9iGpIQEOc1Ka05A%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0; UM_distinctid=169319561a2207-098d1546112786-1333062-144000-169319561a49a; tracknick=%5Cu795E%5Cu52C7%5Cu5341%5Cu4E8C%5Cu751F%5Cu8096; lgc=%5Cu795E%5Cu52C7%5Cu5341%5Cu4E8C%5Cu751F%5Cu8096; uc3=vt3=F8dByEv1RnrwpwbvWUA%3D&id2=UNDVcDrzGd1Mag%3D%3D&nk2=qh7Tuq8EKUklQT%2Bu&lg2=V32FPkk%2Fw0dUvg%3D%3D; _cc_=U%2BGCWk%2F7og%3D%3D; mt=ci=2_1&np=; v=0; cookie2=14303485aef0a08e08100c39c8e8035a; _tb_token_=53b3abe3566e7; uc1=cookie14=UoTZ5bWX5s72Jg%3D%3D; isg=BFJSDnZNWpZclaZnCPBeLNkgoxg-aYGWkoqLsxyrf4XwL_IpBPOmDVhJmosTWc6V\",\n",
    "    'user-agent': ua.random,\n",
    "    'Accept - Encoding':'gzip, deflate',\n",
    "    'Accept-Language':'zh-Hans-CN, zh-Hans; q=0.5',\n",
    "    'Connection':'Keep-Alive'\n",
    "}\n",
    "\n",
    "list_comment = [[]]\n",
    "sig_comment = []\n",
    "count = 0\n",
    "# jsonData = html.text\n",
    "# startLoc = jsonData.find('{')\n",
    "# jsonData = jsonData[startLoc:-1]\n",
    "# jsonData = json.loads(jsonData)\n",
    "pageNum = 120\n",
    "for i in range(100,pageNum):#50为上次中断所采集到的位置\n",
    "    url = 'https://rate.tmall.com/list_detail_rate.htm?itemId=558760911386&spuId=877095771&sellerId=2616970884&order=3&currentPage=%s'%i\n",
    "    print(url)\n",
    "    html=requests.get(url=url,headers=headers)\n",
    "    time.sleep(np.random.uniform(5, 10))\n",
    "    jsonData = html.text\n",
    "    startLoc = jsonData.find('{')\n",
    "    jsonData = jsonData[startLoc:-1]\n",
    "    jsonData = json.loads(jsonData)\n",
    "    for i in range(0, 20):\n",
    "        userId = jsonData['rateDetail']['rateList'][i]['id']#用户ID\n",
    "        userLevel = jsonData['rateDetail']['rateList'][i]['userVipLevel']#用户等级\n",
    "        phone = jsonData['rateDetail']['rateList'][i]['auctionSku']#手机型号\n",
    "        nickName = jsonData['rateDetail']['rateList'][i]['displayUserNick']\n",
    "        buyTime = jsonData['rateDetail']['rateList'][i]['rateDate']\n",
    "        comment = jsonData['rateDetail']['rateList'][i]['rateContent']\n",
    "        sig_comment.append(userId)\n",
    "        sig_comment.append(userLevel)\n",
    "        sig_comment.append(phone)\n",
    "        sig_comment.append(nickName)\n",
    "        sig_comment.append(buyTime)\n",
    "        sig_comment.append(comment)\n",
    "        list_comment.append(sig_comment)\n",
    "        sig_comment = []\n",
    "        count += 1\n",
    "        print(\"第%d条\"%(count))\n",
    "    #html=requests.get(url=url,headers=headers)\n",
    "# print(html.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "存入成功\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "file = open('data/taobaoComment_data.csv','a',newline = '',encoding='gb18030')\n",
    "writer = csv.writer(file)\n",
    "writer.writerow(['用户ID','会员级别','手机参数','昵称','购买时间','评论内容'])\n",
    "for i in range(len(list_comment)):\n",
    "    writer.writerow(list_comment[i])\n",
    "file.close()\n",
    "print(\"存入成功\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
